package org.pblue.tumblarity.task;

import java.io.IOException;
import java.util.Date;
import java.util.logging.Level;
import java.util.logging.Logger;

import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import org.pblue.appengine.AppengineDataException;
import org.pblue.appengine.AppengineDataManager;
import org.pblue.appengine.Cache;
import org.pblue.appengine.CacheList;
import org.pblue.tumblarity.data.bean.Tumblelog;
import org.pblue.tumblarity.exception.TumblarityException;
import org.pblue.tumblarity.service.FetchAllService;
import org.pblue.tumblarity.tools.FollowersPageBean;
import org.pblue.tumblarity.tools.TumblrParser;

import com.google.appengine.api.taskqueue.Queue;
import com.google.appengine.api.taskqueue.QueueFactory;
import com.google.appengine.api.taskqueue.TaskOptions;

/**
 * This is a fetchpage-queue task mapped to /task/fetchpage. It has the following
 * parameters:
 * 
 * tumblelogId (required): the tumblelog id of which' page is to be fetched,
 * 
 * page (required): the page number,
 * 
 * @author mikolaj
 *
 */
public class FetchPage extends HttpServlet {

  private static final long serialVersionUID = 3971144884777643631L;
  
  public static final int SAVE_AFTER_RESET_RETRY_COUNT_TRESHOLD = 20;
  public static final int PAGE_FETCH_RETRY_TRESHOLD = 20;
  
  private static final Logger log = Logger.getLogger(FetchPage.class.getName());

  @Override
  protected synchronized void service(HttpServletRequest pRequest, HttpServletResponse pResponse) throws ServletException, IOException {
    long start = new Date().getTime();
    
    Queue compareQueue = QueueFactory.getQueue(FetchAllService.QUEUE_COMPARE);
    Queue fetchPageQueue = QueueFactory.getQueue(FetchAllService.QUEUE_FETCH_PAGE);
    
    String tumblelogId = pRequest.getParameter(FetchAllService.PARAM_TUMBLELOG_ID);
    String pageParam = pRequest.getParameter(FetchAllService.PARAM_PAGE);

    AppengineDataManager dataManager = AppengineDataManager.open();

    // get page number, and reset tumbelog and exit if unsuccesful
    int page;
    try {
      page = Integer.valueOf(pageParam);
    } catch (NumberFormatException e) {
      log.log(Level.SEVERE, "Page " + pageParam + " of Tumblelog " + tumblelogId + " couldn't be fetched, reseting Tumblelog.", e);
      resetTumblelog(tumblelogId, 0, dataManager);
      dataManager.rollback();
      dataManager.close();
      return;
    }

    log.info("Processing page " + pageParam + " of Tumblelog " + tumblelogId + ".");

    try {
      // retrieve tumblelog from datastore
      dataManager.begin();
      Tumblelog tumblelog = Tumblelog.get(tumblelogId,dataManager);

      if (tumblelog != null) {

        // check that the tumblelog's fetch state is 'processing'
        if (tumblelog.getFetchState() == Tumblelog.FETCH_STATE_PROCESSING) {
          
          // get total number of pages of this tumblelog
          int pageCount = tumblelog.getPageCount();

          boolean finishedPages = false;
          
          // set size of vector storing page html's with the tumblelog, and initialize value
          if (page >= tumblelog.getPageHtmlHash().size()) {
            //tumblelog.getPageHtml().setSize(page + 1);
            //tumblelog.getPageHtml().set(page, new Text(""));
            tumblelog.getPageHtmlHash().add("".hashCode());
          }

          // get previously fetched followers of this page
          CacheList<String> fetchedFollowerIdsOfPage = tumblelog.getFetchedFollowerIdsForPage(page, dataManager);

          // parse the page and extract a list of followers (this call can throw
          // a TumblarityException if the page cannot be fetched from its url, or
          // an IOException and a JDOMException if the SAXParser cannot parse the 
          // html fetched, and a JDOMException if the xpath expressions cannot be
          // evaluated against the html to get the properties of followers
          FollowersPageBean followersPageBean = TumblrParser.getFollowers(tumblelog, page, dataManager);

          // if this is the last page according to the page count property but there 
          // was a next button found on this page, then increment the page count of 
          // the tumblelog, otherwise set finished flag to true
          if (page == pageCount) {
            if (followersPageBean.isNextPage()) {
              pageCount++;
            } else {
              finishedPages = true;
            }
          }              

          // check if returned html equals with the html stored with the tumblelog
          // for this page. if so, then don't process fetched followers
          if (followersPageBean.isNoChange()) {
            log.info("Page " + pageParam + " of Tumblelog " + tumblelogId + " hasn't changed since last fetch, not fetching.");

            // check if tumblelog has zero followers
          } else if (followersPageBean.isNoFollowers()) {
            tumblelog.getPageHtmlHash().set(page, followersPageBean.getHtmlHash());

            // invalidate cached lists of tumblelog
            Cache.remove(Tumblelog.FETCHED_FOLLOWER_LIST_PREFIX + tumblelog.getId());
            Cache.remove(Tumblelog.FETCHED_FOLLOWER_LIST_FOR_PAGE_PREFIX + tumblelog.getId() + "_" + page);

            // remove previously fetched followers for this page
            if (fetchedFollowerIdsOfPage != null && fetchedFollowerIdsOfPage.size() > 0) {
              tumblelog.getFetchedFollowerIds().removeAll(fetchedFollowerIdsOfPage);
            }

            log.info("Tumblelog " + tumblelogId + " has no followers, not fetching.");

            // check if page exists, and if not, then set the number of pages of
            // this tumblelog to the previous page
          } else if (!followersPageBean.isPageExists()) {
            pageCount = page - 1;
            tumblelog.getPageHtmlHash().setSize(page);
            finishedPages = true;
            log.warning("Page " + pageParam + " of Tumblelog " + tumblelogId + " doesn't exist, not fetching.");

            // if followers were fetched then add the retrieved followers to this 
            // tumbelog's fetched followers  list. this fetched followers list always 
            // stores the previously fetched followers of the tumblelog and is 
            // incrementally replaced by the actually fetched ones. it is reconciled 
            // with the real follower list in the compare task, after all pages was fetched.          
          } else if (followersPageBean.getFollowerIds() != null) {
            // store html of this page with the tumblelog
            tumblelog.getPageHtmlHash().set(page, followersPageBean.getHtmlHash());

            // invalidate cached lists of tumblelog
            Cache.remove(Tumblelog.FETCHED_FOLLOWER_LIST_PREFIX + tumblelog.getId());
            Cache.remove(Tumblelog.FETCHED_FOLLOWER_LIST_FOR_PAGE_PREFIX + tumblelog.getId() + "_" + page);

            // remove previously fetched followers for this page
            if (fetchedFollowerIdsOfPage != null && fetchedFollowerIdsOfPage.size() > 0) {
              tumblelog.getFetchedFollowerIds().removeAll(fetchedFollowerIdsOfPage);
            }

            // add newly fetched followers of this page
            tumblelog.getFetchedFollowerIds().addAll(followersPageBean.getFollowerIds());

            log.info("Page " + pageParam + " of Tumblelog " + tumblelogId + " fetched.");
          } else {
            throw new TumblarityException("Fetched page " + page + " of tumblelog " + tumblelogId + " cannot be parsed.");
          }

          // set the fetch retry count to zero, as the fetch was succesful, and set 
          // new page count
          tumblelog.setFetchRetryCount(0);
          tumblelog.setPageCount(pageCount);

          tumblelog.save(dataManager);
          dataManager.commit();

          // queue a task to compare followers and newly fetched followers if all pages
          // were fetched, or queue a task to fetch the next page if not
          if (finishedPages) {
            compareQueue.add(TaskOptions.Builder.withUrl("/task/compare").param(FetchAllService.PARAM_TUMBLELOG_ID, tumblelogId));
          } else {
            fetchPageQueue.add(TaskOptions.Builder.withUrl("/task/fetchpage")
                .param(FetchAllService.PARAM_TUMBLELOG_ID, tumblelogId)
                .param(FetchAllService.PARAM_PAGE, String.valueOf(page + 1))
            );
          }

          log.info("Page " + pageParam + " of Tumblelog " + tumblelogId + " processed in " + (new Date().getTime() - start) + "ms.");

        } else {
          log.warning("Tumblelog " + tumblelogId + " is not being processed.");
        }
        
      } else {
        log.warning("Tumblelog " + tumblelogId + " not found.");
        resetTumblelog(tumblelogId, 0, dataManager);
      }

    // if any errors occur, then log the error message, increment the tumblelog's 
    // retry count and reset the tumblelog by clearing the fetched pages and fetched
    // followers lists and setting the fetch state to 'error'
    } catch (Exception e) {
      log.log(Level.SEVERE, "Page " + pageParam + " of Tumblelog " + tumblelogId + " couldn't be fetched, reseting Tumblelog.", e);
      resetTumblelog(tumblelogId, 0, dataManager);
    } finally {
      dataManager.rollback();
      dataManager.close();
    }
  }
  
  /**
   * This method resets the passed in tumblelog by clearing the fetched pages list 
   * and the fetched followers list, and setting its fetch state property to 'idle'.
   * The it saves the tumblelog, and if cannot be saved then the method calls itself
   * by passing in an incremented save retry counter. If that save retry counter
   * reaches the save retry count limit, then an error message is logged and the
   * tumblelog stays in a 'processing' fetch state, which means it is never fetched
   * again.
   * 
   * @param pTumblelogId The tumblelog's id.
   * @param pSaveRetryCounter The save retry counter, initialized to 0 when the method
   * is first called, then incremented at every recursion.
   */
  private void resetTumblelog(String pTumblelogId, int pSaveRetryCounter, AppengineDataManager pDataManager) {
    try {
      if (!pDataManager.isOpen()) {
        pDataManager.begin();
      }
      Tumblelog tumblelog = Tumblelog.get(pTumblelogId, pDataManager);
      tumblelog.setFetchRetryCount(tumblelog.getFetchRetryCount() + 1);
      tumblelog.setFetchState(Tumblelog.FETCH_STATE_ERROR);
      tumblelog.save(pDataManager);
      pDataManager.commit();
      log.warning("Tumblelog " + pTumblelogId + " has been reset and its fetch state set to ERROR.");
    } catch (AppengineDataException e) {
      log.log(Level.SEVERE, e.getMessage(), e);
      if (pSaveRetryCounter < SAVE_AFTER_RESET_RETRY_COUNT_TRESHOLD) {
        resetTumblelog(pTumblelogId, pSaveRetryCounter + 1, pDataManager);
      } else {
        log.severe("Tumblelog " + pTumblelogId + " couldn't be reset, won't be fetched anymore.");
      }
    }
  }

}
